package com.jvm123.dytt

import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements



def getPageContent(String url, FileWriter writer) {

    int tryTimes = 0

    Document doc = null
    while (doc == null) {
        tryTimes += 1
        doc = Jsoup.connect(url).get()
        if (doc == null) {
            Thread.sleep(2000)
            println "======== has try ${tryTimes} times =========="
        }
        if (tryTimes > 6) {
            println "======== has try ${tryTimes} times and no more try =========="
            return
        }
    }
    // div
    Elements elements = doc.getElementsByClass("tableListMain")
    // tbody
    Element tbody = elements.get(0).getElementsByTag("tbody").get(0)
    // tables
    Elements eleList = tbody.children()
    for (Element tr : eleList.findAll()) {
        try {
            Elements tds = tr.getElementsByTag("td")
            // a
            String a = tds.get(0).getElementsByTag("a").get(0)
            String type = tds.get(1)
            String country = tds.get(2)
            String time = tds.get(4)
            String actors = tds.get(5).getElementsByTag("div").get(0)
            println a
            writer.write("\n")
            writer.write(a.replace("\n", " "))
            writer.write(type.replace("\n", " "))
            writer.write(country.replace("\n", " "))
            writer.write(time.replace("\n", " "))
            writer.write(actors.replace("\n", " "))
        } catch (Exception ex) {
            ex.printStackTrace()
        }
    }
}

File newFile(String fileName) {
    File file = new File(fileName)
    if (!file.exists()) {
        file.createNewFile()
    }
    file
}


File file = newFile("dytt.com.txt")
FileWriter fw = new FileWriter(file)
for (i in 1 .. 2124) { //
    String url = "https://www.dytt.com/fenlei/1-${i}.html"
    getPageContent(url, fw)
    println i
}
fw.close()
